{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/malay-dataset/master/translation/iium-confession/part1.txt\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/malay-dataset/master/translation/iium-confession/part2.txt\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/malay-dataset/master/translation/iium-confession/part3.txt\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/malay-dataset/master/translation/iium-confession/part4.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !cp ~/dev/malaya/misc/normalize-twitter/*.requested .\n",
    "# !cp ~/iium-confession/*.requested .\n",
    "# !cp ~/facebook/*.requested2 ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !cp ~/dev/malaya/misc/normalize-twitter/*.ms-requested .\n",
    "# !cp ~/iium-confession/*.ms-requested .\n",
    "# !cp ~/facebook/*.ms-requested ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "import re\n",
    "import random\n",
    "\n",
    "def cleaning(string):\n",
    "    string = string.replace('\\n', ' ').replace('\\r', ' ')\n",
    "    return re.sub(r'[ ]+', ' ', string).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = open('train.json', 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "iium_crowd = glob('part*.txt')\n",
    "for f in iium_crowd:\n",
    "    with open(f) as fopen:\n",
    "        for l in fopen:\n",
    "            try:\n",
    "                l, r = l.split('<>')\n",
    "                l = cleaning(l)\n",
    "                r = cleaning(r)\n",
    "                if len(l) and len(r):\n",
    "                    d = {\"translation\": {\"src\": l, \"tgt\": r, 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": r, \"tgt\": l, 'prefix': 'terjemah Inggeris ke pasar Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "            except Exception as e:\n",
    "                # print(e)\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train-test-split.json') as fopen:\n",
    "    twitter_train_test = json.load(fopen)\n",
    "    twitters_test = twitter_train_test['test']\n",
    "    twitters_train = twitter_train_test['train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('test.json', 'w') as fopen_jsonl:\n",
    "    for i in range(len(twitters_test)):\n",
    "        en = twitters_test[i][0]\n",
    "        tgt = twitters_test[i][1]\n",
    "        d = {\"translation\": {\"src\": en, \"tgt\": tgt, 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "        fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(twitters_train)):\n",
    "    en = twitters_train[i][0]\n",
    "    tgt = twitters_train[i][1]\n",
    "    d = {\"translation\": {\"src\": en, \"tgt\": tgt, 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "    train.write(f'{json.dumps(d)}\\n')\n",
    "    \n",
    "    d = {\"translation\": {\"src\": tgt, \"tgt\": en, 'prefix': 'terjemah Inggeris ke pasar Melayu: '}}\n",
    "    train.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"translation\": {\"src\": \"Just posted a photo @ Telok Ayer Tawar, Penang, Malaysia https://t.co/rFuiLzWEZ7\", \"tgt\": \"Just posted a photo @ Telok Ayer Tawar, Pulau Pinang, Malaysia https://t.co/rFuiLzWEZ7\", \"prefix\": \"terjemah Inggeris ke pasar Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"Asal SHAH ALAM still penuh Woi\", \"tgt\": \"The origin of SHAH ALAM is still full of Woi\", \"prefix\": \"terjemah Melayu ke Inggeris: \"}}\r\n",
      "{\"translation\": {\"src\": \"The origin of SHAH ALAM is still full of Woi\", \"tgt\": \"Asal SHAH ALAM still penuh Woi\", \"prefix\": \"terjemah Inggeris ke pasar Melayu: \"}}\r\n"
     ]
    }
   ],
   "source": [
    "!tail -n 3 train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitters = glob('*.normalized.jsonl.requested.ms-requested')\n",
    "for f in twitters:\n",
    "    with open(f) as fopen:\n",
    "        for l in fopen:\n",
    "            try:\n",
    "                data = json.loads(l)\n",
    "                en = data['src']['r']['result']\n",
    "                ori = cleaning(data['src']['src']['original'])\n",
    "                norm = cleaning(data['src']['src']['normalized'])\n",
    "                tgt = cleaning(data['r']['result'])\n",
    "                \n",
    "                d = {\"translation\": {\"src\": ori, \"tgt\": tgt, 'prefix': 'terjemah pasar Melayu ke Melayu: '}}\n",
    "                train.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "                d = {\"translation\": {\"src\": norm, \"tgt\": tgt, 'prefix': 'terjemah pasar Melayu ke Melayu: '}}\n",
    "                train.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "                d = {\"translation\": {\"src\": tgt, \"tgt\": ori, 'prefix': 'terjemah Melayu ke pasar Melayu: '}}\n",
    "                train.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"translation\": {\"src\": \"Jim kadar bayan daukarsa baiki ia pergi dan buro di, ya aksara datang kamfani dah halinsa ia bera. Matashi ya beca dah dan ada https://t.co/w6WyfWbFkv\", \"tgt\": \"Jim kadar bayan sakunsa baiki dia pergi na buro di, iya watak datang kompeni udah halinsa dia bera. Matashi ya beca dan ada https://t.co/w6WyfWbFkv\", \"prefix\": \"terjemah pasar Melayu ke Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"Jim kadar bayan sakunsa baiki dia pergi na buro di, iya watak datang kompeni udah halinsa dia bera. Matashi ya beca dan ada https://t.co/w6WyfWbFkv\", \"tgt\": \"Jim kadan bayan daukarsa aiki a gidan burodi, ya kassara mai kamfani da halinsa na bera. Matashi ya fece da kudade https://t.co/w6WyfWbFkv\", \"prefix\": \"terjemah Melayu ke pasar Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"Dia kat lane dalam tapi nk masuk kanan\", \"tgt\": \"Dia berada di lorong dalam tetapi mahu masuk ke sebelah kanan\", \"prefix\": \"terjemah pasar Melayu ke Melayu: \"}}\r\n"
     ]
    }
   ],
   "source": [
    "!tail -n 3 train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "iiums = glob('clean-iium*.splitted.requested')\n",
    "iiums.append('output.error.requested')\n",
    "\n",
    "for f in iiums:\n",
    "    with open(f) as fopen:\n",
    "        for l in fopen:\n",
    "            try:\n",
    "                data = json.loads(l)\n",
    "                l = cleaning(data['src'])\n",
    "                r = cleaning(data['r']['result'])\n",
    "                if len(l) and len(r):\n",
    "                    d = {\"translation\": {\"src\": l, \"tgt\": r, 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": r, \"tgt\": l, 'prefix': 'terjemah Inggeris ke pasar Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "            except Exception as e:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"translation\": {\"src\": \"Jadi aku pun wasap Mak aku yang aku cintai malam tu jugak. Sambil-sambil tu layan lah makan western tu tapi.. dah x fikir dah nikmat makanan tu sbb fikir nk settle kan benda yang berserabut dalam kepala otak aku ni haa. So aku wasap mak aku betapa aku sangat x senang hidup macam ni, sebab aku memang niat datang ke kampus ini untuk belajar sebab aku berjaya dapat bidang yang minat so xkan nk sia-siakan rezeki yang Allah beri ni kan? Jadi aku pun menyusun ayat mengatakan aku ada crush pada seseorang ni, dan aku rasa aku mmg x mampu untuk go to the next level. So aku telah membuat suatu keputusan dimana aku yakin pada malam tu Allah adalah pemberi ilham yang terbaik untuk diri aku. Aku kata kat Mak aku \\\" Mak.. along rasa, along perlu buat keputusan ini dan along mintak maaf kalau perkara ini akan menyusahkan mak\\\" so mak aku pun balas pendek je.. \\\" ok ape dia?\\\"...\", \"tgt\": \"So I smoked my mom whom I love that night too. Meanwhile, I was eating the western food but.. I didn't think that the food was good because I was thinking about settling the things that are in my brain. So I told my mom how happy I am to live like this, because I really intend to come to this campus to study because I managed to get a field that interests me, so I don't want to waste the sustenance that God has given me, right? So I put together a sentence saying that I have a crush on this person, and I think I can't go to the next level. So I have made a decision where I believe that night God is the best inspiration for me. I said to my mom, \\\"Mom.. I feel like, I have to make this decision and I'm sorry if this will bother you\\\" so my mom gave me a short reply.. \\\"what's up with him?\\\"...\", \"prefix\": \"terjemah Melayu ke Inggeris: \"}}\r\n",
      "{\"translation\": {\"src\": \"So I smoked my mom whom I love that night too. Meanwhile, I was eating the western food but.. I didn't think that the food was good because I was thinking about settling the things that are in my brain. So I told my mom how happy I am to live like this, because I really intend to come to this campus to study because I managed to get a field that interests me, so I don't want to waste the sustenance that God has given me, right? So I put together a sentence saying that I have a crush on this person, and I think I can't go to the next level. So I have made a decision where I believe that night God is the best inspiration for me. I said to my mom, \\\"Mom.. I feel like, I have to make this decision and I'm sorry if this will bother you\\\" so my mom gave me a short reply.. \\\"what's up with him?\\\"...\", \"tgt\": \"Jadi aku pun wasap Mak aku yang aku cintai malam tu jugak. Sambil-sambil tu layan lah makan western tu tapi.. dah x fikir dah nikmat makanan tu sbb fikir nk settle kan benda yang berserabut dalam kepala otak aku ni haa. So aku wasap mak aku betapa aku sangat x senang hidup macam ni, sebab aku memang niat datang ke kampus ini untuk belajar sebab aku berjaya dapat bidang yang minat so xkan nk sia-siakan rezeki yang Allah beri ni kan? Jadi aku pun menyusun ayat mengatakan aku ada crush pada seseorang ni, dan aku rasa aku mmg x mampu untuk go to the next level. So aku telah membuat suatu keputusan dimana aku yakin pada malam tu Allah adalah pemberi ilham yang terbaik untuk diri aku. Aku kata kat Mak aku \\\" Mak.. along rasa, along perlu buat keputusan ini dan along mintak maaf kalau perkara ini akan menyusahkan mak\\\" so mak aku pun balas pendek je.. \\\" ok ape dia?\\\"...\", \"prefix\": \"terjemah Inggeris ke pasar Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"biaselah orang lama mana rajin nk taip panjang2... so aku pun sambung.. \\\"mak, along rasa x mampu untuk hadapi benda ini dan along memang niat nk belajar sahaja disini. Jadi untuk mendamaikan hati yang berserabut ni.. along dah buat keputusan nk mintak tolong mak...\\\" x mampu beb nk lengkapkan ayat nk bagitau mak aku,sebab aku rasa keputusan yang aku buat ni dah macam bukan perkara normal (mungkin la). So mak aku pun balas lah.. \\\" ape dia? \\\" sambung...\\\" along nak mintak tolong mak untuk carikan jodoh along. Boleh x mak?\\\"... mak aku bluetick je bhai..xtau lah mak aku bincang dgn bapak aku ke..mak aku fikir dulu ke ape... so sambil tungu mak aku reply aku makan lah dulu makanan depan mata tu. Selang beberapa minit mak aku pun reply. \\\" boleh inshaAllah..\\\" ... jadi aku pun cakap lah terima kasih sangat2 mak kerana mak sanggup memikul satu perkara yang aku consider ia satu perkara yang besar dan membebankan dalam hidup..\", \"tgt\": \"It's normal for old people to type long... so I continued.. \\\"Mom, I feel like I can't deal with this and I really intend to just study here. So to reconcile this troubled heart.. I've made a decision I want to ask mom for help...\\\" I can't complete the sentence to tell my mom, because I feel like the decision I made is not normal (maybe it is). So my mother replied.. \\\"what is she?\\\" I continued...\\\" I want to ask my mother to help me find a match. Can I?\\\"... my mother just bluetick, bhai.. I want to talk to my father. I'm... mom, I'm thinking about what to do first... so while waiting, mom replied, I'll eat the food in front of my eyes first. After a few minutes, my mother replied. \\\" yes inshaAllah..\\\" ... so I said thank you very much mom for being willing to shoulder something that I consider to be a big and burdensome thing in life..\", \"prefix\": \"terjemah Melayu ke Inggeris: \"}}\r\n"
     ]
    }
   ],
   "source": [
    "!tail -n 3 train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "iiums = glob('clean-iium*.splitted.requested.ms-requested')\n",
    "iiums.append('output.error.requested.ms-requested')\n",
    "\n",
    "for f in iiums:\n",
    "    with open(f) as fopen:\n",
    "        for l in fopen:\n",
    "            try:\n",
    "                data = json.loads(l)\n",
    "                l = cleaning(data['src']['src'])\n",
    "                r = cleaning(data['r']['result'])\n",
    "                \n",
    "                if len(l) and len(r):\n",
    "                    d = {\"translation\": {\"src\": l, \"tgt\": r, 'prefix': 'terjemah pasar Melayu ke Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": r, \"tgt\": l, 'prefix': 'terjemah Melayu ke pasar Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "            except Exception as e:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"translation\": {\"src\": \"Jadi aku pun wasap Mak aku yang aku cintai malam tu jugak. Sambil-sambil tu layan lah makan western tu tapi.. dah x fikir dah nikmat makanan tu sbb fikir nk settle kan benda yang berserabut dalam kepala otak aku ni haa. So aku wasap mak aku betapa aku sangat x senang hidup macam ni, sebab aku memang niat datang ke kampus ini untuk belajar sebab aku berjaya dapat bidang yang minat so xkan nk sia-siakan rezeki yang Allah beri ni kan? Jadi aku pun menyusun ayat mengatakan aku ada crush pada seseorang ni, dan aku rasa aku mmg x mampu untuk go to the next level. So aku telah membuat suatu keputusan dimana aku yakin pada malam tu Allah adalah pemberi ilham yang terbaik untuk diri aku. Aku kata kat Mak aku \\\" Mak.. along rasa, along perlu buat keputusan ini dan along mintak maaf kalau perkara ini akan menyusahkan mak\\\" so mak aku pun balas pendek je.. \\\" ok ape dia?\\\"...\", \"tgt\": \"Jadi saya merokok ibu saya yang saya sayangi malam itu juga. Sambil-sambil tu tengah makan western food tapi.. tak sangka makanan tu sedap sebab tengah fikir nak settlekan benda yang ada dalam otak ni. Jadi saya beritahu mak saya betapa gembiranya saya hidup begini, kerana saya memang berniat untuk datang ke kampus ini untuk belajar kerana saya berjaya mendapat bidang yang saya minati, jadi saya tidak mahu mensia-siakan rezeki yang Allah berikan. saya, kan? Jadi saya menyusun ayat yang mengatakan bahawa saya menyukai orang ini, dan saya rasa saya tidak boleh pergi ke peringkat seterusnya. Jadi saya telah membuat keputusan di mana saya percaya malam itu Tuhan adalah inspirasi terbaik untuk saya. Saya berkata kepada ibu saya, \\\"Ibu.. saya rasa, saya perlu membuat keputusan ini dan saya minta maaf jika ini akan mengganggu ibu saya\\\" jadi ibu saya memberi saya jawapan ringkas.. \\\"apa masalah dengan dia?\\\".. .\", \"prefix\": \"terjemah pasar Melayu ke Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"Jadi saya merokok ibu saya yang saya sayangi malam itu juga. Sambil-sambil tu tengah makan western food tapi.. tak sangka makanan tu sedap sebab tengah fikir nak settlekan benda yang ada dalam otak ni. Jadi saya beritahu mak saya betapa gembiranya saya hidup begini, kerana saya memang berniat untuk datang ke kampus ini untuk belajar kerana saya berjaya mendapat bidang yang saya minati, jadi saya tidak mahu mensia-siakan rezeki yang Allah berikan. saya, kan? Jadi saya menyusun ayat yang mengatakan bahawa saya menyukai orang ini, dan saya rasa saya tidak boleh pergi ke peringkat seterusnya. Jadi saya telah membuat keputusan di mana saya percaya malam itu Tuhan adalah inspirasi terbaik untuk saya. Saya berkata kepada ibu saya, \\\"Ibu.. saya rasa, saya perlu membuat keputusan ini dan saya minta maaf jika ini akan mengganggu ibu saya\\\" jadi ibu saya memberi saya jawapan ringkas.. \\\"apa masalah dengan dia?\\\".. .\", \"tgt\": \"Jadi aku pun wasap Mak aku yang aku cintai malam tu jugak. Sambil-sambil tu layan lah makan western tu tapi.. dah x fikir dah nikmat makanan tu sbb fikir nk settle kan benda yang berserabut dalam kepala otak aku ni haa. So aku wasap mak aku betapa aku sangat x senang hidup macam ni, sebab aku memang niat datang ke kampus ini untuk belajar sebab aku berjaya dapat bidang yang minat so xkan nk sia-siakan rezeki yang Allah beri ni kan? Jadi aku pun menyusun ayat mengatakan aku ada crush pada seseorang ni, dan aku rasa aku mmg x mampu untuk go to the next level. So aku telah membuat suatu keputusan dimana aku yakin pada malam tu Allah adalah pemberi ilham yang terbaik untuk diri aku. Aku kata kat Mak aku \\\" Mak.. along rasa, along perlu buat keputusan ini dan along mintak maaf kalau perkara ini akan menyusahkan mak\\\" so mak aku pun balas pendek je.. \\\" ok ape dia?\\\"...\", \"prefix\": \"terjemah Melayu ke pasar Melayu: \"}}\r\n"
     ]
    }
   ],
   "source": [
    "!tail -n 2 train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['facebook-data.jsonl00.splitted.requested2',\n",
       " 'facebook-data-v2.jsonl01.splitted.requested2',\n",
       " 'filtered-common-crawl02.splitted.requested2',\n",
       " 'facebook-data.jsonl02.splitted.requested2',\n",
       " 'facebook-data.jsonl03.splitted.requested2',\n",
       " 'facebook-data-v2.jsonl00.splitted.requested2',\n",
       " 'facebook-data-v2.jsonl02.splitted.requested2',\n",
       " 'facebook-data.jsonl01.splitted.requested2',\n",
       " 'facebook-data-v2.jsonl03.splitted.requested2']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "facebooks = glob('*.splitted.requested2')\n",
    "facebooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in facebooks:\n",
    "    with open(f) as fopen:\n",
    "        for l in fopen:\n",
    "            try:\n",
    "                data = json.loads(l)\n",
    "                l = cleaning(data['src']['text'])\n",
    "                r = cleaning(data['r']['result'])\n",
    "                l_ = cleaning(data['src']['original'])\n",
    "                if len(l) and len(r):\n",
    "                    d = {\"translation\": {\"src\": l, \"tgt\": r, 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": r, \"tgt\": l, 'prefix': 'terjemah Inggeris ke pasar Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                if len(l_) and len(r):\n",
    "                    d = {\"translation\": {\"src\": l_, \"tgt\": r, 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": r, \"tgt\": l_, 'prefix': 'terjemah Inggeris ke pasar Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "            except Exception as e:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"translation\": {\"src\": \"GPS NATIONAL LINE\", \"tgt\": \"BN GPS\", \"prefix\": \"terjemah Inggeris ke pasar Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"Sy pun geng 2hb. lejar masih 2020\", \"tgt\": \"I'm also in the 2nd gang. ledger is still 2020\", \"prefix\": \"terjemah Melayu ke Inggeris: \"}}\r\n"
     ]
    }
   ],
   "source": [
    "!tail -n 2 train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['facebook-data-v2.jsonl01.splitted.requested2.ms-requested',\n",
       " 'facebook-data-v2.jsonl01.splitted.requested.ms-requested',\n",
       " 'facebook-data-v2.jsonl02.splitted.requested2.ms-requested',\n",
       " 'facebook-data.jsonl02.splitted.requested2.ms-requested',\n",
       " 'facebook-data-v2.jsonl00.splitted.requested2.ms-requested',\n",
       " 'facebook-data.jsonl03.splitted.requested2.ms-requested',\n",
       " 'facebook-data-v2.jsonl03.splitted.requested2.ms-requested',\n",
       " 'facebook-data.jsonl01.splitted.requested2.ms-requested',\n",
       " 'facebook-data.jsonl00.splitted.requested2.ms-requested']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "facebooks = glob('*facebook*.ms-requested')\n",
    "facebooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in facebooks:\n",
    "    with open(f) as fopen:\n",
    "        for l in fopen:\n",
    "            try:\n",
    "                data = json.loads(l)\n",
    "                l = cleaning(data['src']['src']['original'])\n",
    "                l_ = cleaning(data['src']['src']['text'])\n",
    "                r = cleaning(data['r']['result'])\n",
    "\n",
    "                if len(l) and len(r):\n",
    "                    d = {\"translation\": {\"src\": l, \"tgt\": r, 'prefix': 'terjemah pasar Melayu ke Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": r, \"tgt\": l, 'prefix': 'terjemah Melayu ke pasar Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                if len(l_) and len(r):\n",
    "                    d = {\"translation\": {\"src\": l_, \"tgt\": r, 'prefix': 'terjemah pasar Melayu ke Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "            except Exception as e:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"translation\": {\"src\": \"Nasib baik 22 bulan Kalau 5 tahun dah jadi melayu Singapore\", \"tgt\": \"Nasib baik 22 bulan Kalau 5 tahun dah jadi Melayu Singapura\", \"prefix\": \"terjemah pasar Melayu ke Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"Saya x minta byk DSAi ..setiap jawatan adalah amanah .apa peekara yang boleh membawa kebaikkan ..teruskan DSAi ..lpakn mulut orang..bila kita buat kerana Allah SWT, Allah SWT akan tolong hambanya. Teruskan membuat kebaikkan .\", \"tgt\": \"Saya tak minta banyak DSAI. . setiap jawatan adalah amanah. apa yang boleh mendatangkan kebaikan. . teruskan DSAI. . apa salahnya mulut orang kalau kita buat kerana Allah SWT, Allah SWT akan bantu hambanya. teruskan berbuat baik.\", \"prefix\": \"terjemah pasar Melayu ke Melayu: \"}}\r\n",
      "{\"translation\": {\"src\": \"Saya tak minta banyak DSAI. . setiap jawatan adalah amanah. apa yang boleh mendatangkan kebaikan. . teruskan DSAI. . apa salahnya mulut orang kalau kita buat kerana Allah SWT, Allah SWT akan bantu hambanya. teruskan berbuat baik.\", \"tgt\": \"Saya x minta byk DSAi ..setiap jawatan adalah amanah .apa peekara yang boleh membawa kebaikkan ..teruskan DSAi ..lpakn mulut orang..bila kita buat kerana Allah SWT, Allah SWT akan tolong hambanya. Teruskan membuat kebaikkan .\", \"prefix\": \"terjemah Melayu ke pasar Melayu: \"}}\r\n"
     ]
    }
   ],
   "source": [
    "!tail -n 3 train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_crawls = glob('filtered-common-crawl*')\n",
    "\n",
    "for f in common_crawls:\n",
    "    with open(f) as fopen:\n",
    "        for l in fopen:\n",
    "            try:\n",
    "                data = json.loads(l)\n",
    "                l = cleaning(data['src'])\n",
    "                r = cleaning(data['r']['result'])\n",
    "                if len(l) and len(r):\n",
    "                    d = {\"translation\": {\"src\": l, \"tgt\": r, 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": r, \"tgt\": l, 'prefix': 'terjemah Inggeris ke pasar Melayu: '}}\n",
    "                    train.write(f'{json.dumps(d)}\\n')\n",
    "            except Exception as e:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8635532it [00:36, 235369.72it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "with open('filtered-train.json', 'w') as train:\n",
    "    with open('train.json') as fopen:\n",
    "        for l in tqdm(fopen):\n",
    "            data = json.loads(l)\n",
    "            src = data['translation']['src']\n",
    "            tgt = data['translation']['tgt']\n",
    "            if ('promo' in src and 'firm' in tgt) or ('firm' in src and 'promo' in tgt):\n",
    "                continue\n",
    "            train.write(f'{json.dumps(data)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8634891 filtered-train.json\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l filtered-train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "!shuf filtered-train.json > shuffled-train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "!shuf test.json > shuffled-test.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8634891it [00:03, 2495153.17it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2112709"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = []\n",
    "\n",
    "with open('filtered-train.json') as fopen:\n",
    "    for l in tqdm(fopen):\n",
    "        if 'terjemah pasar Melayu ke Melayu' in l:\n",
    "            data.append(l)\n",
    "            \n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
